{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1: Import Packages\n",
    "Modify the system path and load the corresponding packages and functions "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "from pathlib import Path\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "ROOT = str(Path(\"__file__\").resolve().parents[1])\n",
    "sys.path.append(ROOT)\n",
    "import torch\n",
    "import argparse\n",
    "import os.path as osp\n",
    "from mmcv import Config\n",
    "from trademaster.utils import replace_cfg_vals\n",
    "from trademaster.nets.builder import build_net\n",
    "from trademaster.environments.builder import build_environment\n",
    "from trademaster.datasets.builder import build_dataset\n",
    "from trademaster.agents.builder import build_agent\n",
    "from trademaster.optimizers.builder import build_optimizer\n",
    "from trademaster.losses.builder import build_loss\n",
    "from trademaster.trainers.builder import build_trainer\n",
    "from trademaster.transition.builder import build_transition"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Load Configs\n",
    "Load default config from the folder `configs/algorithmic_trading/algorithmic_trading_BTC_dqn_dqn_adam_mse.py`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = argparse.ArgumentParser(description='Download Alpaca Datasets')\n",
    "parser.add_argument(\"--config\", default=osp.join(ROOT, \"configs\", \"algorithmic_trading\", \"algorithmic_trading_BTC_dqn_dqn_adam_mse.py\"),\n",
    "                    help=\"download datasets config file path\")\n",
    "parser.add_argument(\"--task_name\", type=str, default=\"train\")\n",
    "parser.add_argument(\"--test_style\", type=str, default='-1')\n",
    "args = parser.parse_args([])\n",
    "cfg = Config.fromfile(args.config)\n",
    "task_name = args.task_name\n",
    "\n",
    "cfg = replace_cfg_vals(cfg)\n",
    "# update test style\n",
    "cfg.data.update({'test_style': args.test_style})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Config (path: /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/configs/algorithmic_trading/algorithmic_trading_BTC_dqn_dqn_adam_mse.py): {'data': {'type': 'AlgorithmicTradingDataset', 'data_path': 'data/algorithmic_trading/BTC', 'train_path': 'data/algorithmic_trading/BTC/train.csv', 'valid_path': 'data/algorithmic_trading/BTC/valid.csv', 'test_path': 'data/algorithmic_trading/BTC/test.csv', 'test_style_path': 'data/algorithmic_trading/BTC/test_labeled_3_24_-0.15_0.15.csv', 'tech_indicator_list': ['high', 'low', 'open', 'close', 'adjcp', 'zopen', 'zhigh', 'zlow', 'zadjcp', 'zclose', 'zd_5', 'zd_10', 'zd_15', 'zd_20', 'zd_25', 'zd_30'], 'backward_num_day': 5, 'forward_num_day': 5, 'test_style': '-1'}, 'environment': {'type': 'AlgorithmicTradingEnvironment'}, 'agent': {'type': 'AlgorithmicTradingDQN', 'max_step': 12345, 'reward_scale': 1, 'repeat_times': 1, 'gamma': 0.9, 'batch_size': 64, 'clip_grad_norm': 3.0, 'soft_update_tau': 0, 'state_value_tau': 0.005}, 'trainer': {'type': 'AlgorithmicTradingTrainer', 'epochs': 20, 'work_dir': 'work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse', 'seeds_list': (12345,), 'batch_size': 64, 'horizon_len': 1024, 'buffer_size': 1000000.0, 'num_threads': 8, 'if_remove': False, 'if_discrete': True, 'if_off_policy': True, 'if_keep_save': True, 'if_over_write': False, 'if_save_buffer': False}, 'loss': {'type': 'MSELoss'}, 'optimizer': {'type': 'Adam', 'lr': 0.001}, 'act': {'type': 'QNet', 'state_dim': 82, 'action_dim': 3, 'dims': (64, 32), 'explore_rate': 0.25}, 'cri': None, 'transition': {'type': 'Transition'}, 'task_name': 'algorithmic_trading', 'dataset_name': 'BTC', 'optimizer_name': 'adam', 'loss_name': 'mse', 'net_name': 'dqn', 'agent_name': 'dqn', 'work_dir': 'work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse', 'batch_size': 64}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cfg"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Build Dataset\n",
    "Build datasets from cfg defined above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = build_dataset(cfg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<trademaster.datasets.algorithmic_trading.dataset.AlgorithmicTradingDataset at 0x7f7222541890>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4: Build Reinforcement Learning Environments\n",
    "Build environments based on cfg and previously-defined dataset\n",
    "\n",
    "A style-test is provided as an option to test the algorithm's performance under different market conditions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_environment = build_environment(cfg, default_args=dict(dataset=dataset, task=\"train\"))\n",
    "valid_environment = build_environment(cfg, default_args=dict(dataset=dataset, task=\"valid\"))\n",
    "test_environment = build_environment(cfg, default_args=dict(dataset=dataset, task=\"test\"))\n",
    "if task_name.startswith(\"style_test\"):\n",
    "        test_style_environments = []\n",
    "        for i, path in enumerate(dataset.test_style_paths):\n",
    "            test_style_environments.append(build_environment(cfg, default_args=dict(dataset=dataset, task=\"test_style\",\n",
    "                                                                                    style_test_path=path,\n",
    "                                                                                    task_index=i)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<trademaster.environments.algorithmic_trading.environment.AlgorithmicTradingEnvironment at 0x7f7222553350>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<trademaster.environments.algorithmic_trading.environment.AlgorithmicTradingEnvironment at 0x7f72225532d0>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<trademaster.environments.algorithmic_trading.environment.AlgorithmicTradingEnvironment at 0x7f7222553310>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_environment"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5: Build Net \n",
    "Update information about the state and action dimension in the config and create nets and optimizer for DQN\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "action_dim = train_environment.action_dim\n",
    "state_dim = train_environment.state_dim\n",
    "\n",
    "cfg.act.update(dict(action_dim=action_dim, state_dim=state_dim))\n",
    "act = build_net(cfg.act)\n",
    "act_optimizer = build_optimizer(cfg, default_args=dict(params=act.parameters()))\n",
    "if cfg.cri:\n",
    "    cfg.cri.update(dict(action_dim=action_dim, state_dim=state_dim))\n",
    "    cri = build_net(cfg.cri)\n",
    "    cri_optimizer = build_optimizer(cfg, default_args=dict(params=cri.parameters()))\n",
    "else:\n",
    "    cri = None\n",
    "    cri_optimizer = None"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6: Build Loss\n",
    "Build loss from config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = build_loss(cfg)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 7: Build Transition\n",
    "Build transition from config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "transition = build_transition(cfg)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 8: Build Agent\n",
    "Build agent from config and detect device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "agent = build_agent(cfg, default_args=dict(action_dim = action_dim,\n",
    "                                               state_dim = state_dim,\n",
    "                                               act = act,\n",
    "                                               cri = cri,\n",
    "                                               act_optimizer = act_optimizer,\n",
    "                                               cri_optimizer = cri_optimizer,\n",
    "                                               criterion = criterion,\n",
    "                                               transition = transition,\n",
    "                                               device=device))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 9: Build Trainer\n",
    "Build trainer from config and create work directionary to save the result, model and config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Arguments Keep work_dir: /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse\n"
     ]
    }
   ],
   "source": [
    "if task_name.startswith(\"style_test\"):\n",
    "    trainers = []\n",
    "    for env in test_style_environments:\n",
    "        trainers.append(build_trainer(cfg, default_args=dict(train_environment=train_environment,\n",
    "                                                                valid_environment=valid_environment,\n",
    "                                                                test_environment=env,\n",
    "                                                                agent=agent,\n",
    "                                                                device=device)))\n",
    "else:\n",
    "    trainer = build_trainer(cfg, default_args=dict(train_environment=train_environment,\n",
    "                                                    valid_environment=valid_environment,\n",
    "                                                    test_environment=test_environment,\n",
    "                                                    agent=agent,\n",
    "                                                    device=device))\n",
    "\n",
    "cfg.dump(osp.join(ROOT, cfg.work_dir, osp.basename(args.config)))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 10: Train the Trainer\n",
    "Train the trainer based on the config and get results from workdir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Episode: [1/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -344.513937% |  -0.000214  | 6862.304186 |   6.214777   | -27423.674698 |   -0.556031   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode: [1/20]\n",
      "+---------------+-------------+-------------+--------------+--------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown | Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+--------------+---------------+\n",
      "|   36.243173%  |   0.001800  | 4108.651796 |   0.583186   | 62313.081138 |    0.551188   |\n",
      "+---------------+-------------+-------------+--------------+--------------+---------------+\n",
      "Valid Episode Reward Sum: 56390.914075\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00001.pth\n",
      "Train Episode: [2/20]\n",
      "+---------------+-------------+--------------+--------------+----------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio  | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+----------------+---------------+\n",
      "|  2550.222626% |   0.000294  | 74654.187304 |   0.984371   | 2592355.042556 |    0.737448   |\n",
      "+---------------+-------------+--------------+--------------+----------------+---------------+\n",
      "Valid Episode: [2/20]\n",
      "+---------------+-------------+-------------+--------------+--------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown | Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+--------------+---------------+\n",
      "|   38.089801%  |   0.002813  | 2762.654677 |   0.456196   | 83702.883876 |    0.945576   |\n",
      "+---------------+-------------+-------------+--------------+--------------+---------------+\n",
      "Valid Episode Reward Sum: 57332.169254\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00002.pth\n",
      "Train Episode: [3/20]\n",
      "+---------------+-------------+--------------+--------------+----------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio  | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+----------------+---------------+\n",
      "|  2129.816364% |   0.000301  | 60992.463565 |   0.990535   | 2151453.831529 |    0.761704   |\n",
      "+---------------+-------------+--------------+--------------+----------------+---------------+\n",
      "Valid Episode: [3/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -230.134541% |  -0.001800  | 4108.651796 |   2.562529   | -14181.348608 |   -0.819081   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -56390.914075\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00003.pth\n",
      "Train Episode: [4/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| -2334.279839% |  -0.000274  | 62958.198404 |  24.260127   | -82729.892836 |   -0.704792   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [4/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -230.064518% |  -0.001804  | 4108.595508 |   2.561829   | -14212.530944 |   -0.820658   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -56365.177630\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00004.pth\n",
      "Train Episode: [5/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| -1641.409323% |  -0.000287  | 43526.174039 |  14.819907   | -97857.697010 |   -0.740099   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [5/20]\n",
      "+---------------+-------------+------------+--------------+--------------+---------------+\n",
      "| Profit Margin | Sharp Ratio | Volatility | Max Drawdown | Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+------------+--------------+--------------+---------------+\n",
      "|   -0.000000%  |   0.000000  |  0.000000  |   0.000000   |   0.000000   |      0nan     |\n",
      "+---------------+-------------+------------+--------------+--------------+---------------+\n",
      "Valid Episode Reward Sum: 0.000000\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00005.pth\n",
      "Train Episode: [6/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| -1691.332913% |  -0.000290  | 46229.282308 |  36.828678   | -42306.683350 |   -0.752619   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [6/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -227.824723% |  -0.001990  | 3952.099453 |   2.539431   | -15219.010810 |   -0.887165   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -60502.612906\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00006.pth\n",
      "Train Episode: [7/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| -2174.481852% |  -0.000292  | 59661.553550 |  47.345066   | -42758.782985 |   -0.759986   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [7/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -176.839873% |  -0.001765  | 3528.124043 |   2.195090   | -13939.332801 |   -0.796095   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -52680.509863\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00007.pth\n",
      "Train Episode: [8/20]\n",
      "+---------------+-------------+--------------+--------------+----------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio  | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+----------------+---------------+\n",
      "|  1885.983659% |   0.000443  | 36656.692362 |   0.972584   | 1941130.966521 |    1.154473   |\n",
      "+---------------+-------------+--------------+--------------+----------------+---------------+\n",
      "Valid Episode: [8/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -230.134541% |  -0.001800  | 4108.651796 |   2.562529   | -14181.348608 |   -0.819081   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -56390.914075\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00008.pth\n",
      "Train Episode: [9/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| -2724.499907% |  -0.000291  | 74498.020183 |  52.412219   | -48016.107359 |   -0.744638   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [9/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -230.134541% |  -0.001800  | 4108.651796 |   2.562529   | -14181.348608 |   -0.819081   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -56390.914075\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00009.pth\n",
      "Train Episode: [10/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| -2031.685492% |  -0.000290  | 54747.564770 |  43.636693   | -42363.764305 |   -0.757473   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [10/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -230.134541% |  -0.001800  | 4108.651796 |   2.562529   | -14181.348608 |   -0.819081   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -56390.914075\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00010.pth\n",
      "Train Episode: [11/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| -2399.121800% |  -0.000292  | 64957.047457 |  51.684147   | -42618.599118 |   -0.761166   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [11/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -230.053823% |  -0.001886  | 4043.229698 |   2.569442   | -14579.510155 |   -0.866852   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -57221.583660\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00011.pth\n",
      "Train Episode: [12/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| -1481.424033% |  -0.000284  | 39030.031859 |  31.315723   | -41117.470365 |   -0.737378   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [12/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -230.134541% |  -0.001800  | 4108.651796 |   2.562529   | -14181.348608 |   -0.819081   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -56390.914075\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00012.pth\n",
      "Train Episode: [13/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "|  -788.802406% |  -0.000152  | 19854.508607 |   6.946587   | -50441.493440 |   -0.380641   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [13/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -170.396222% |  -0.002120  | 3555.765470 |   2.254471   | -16430.348220 |   -0.931926   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -60797.589909\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00013.pth\n",
      "Train Episode: [14/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| -1499.160477% |  -0.000287  | 39627.327133 |  30.212022   | -43745.116737 |   -0.735405   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [14/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -169.507466% |  -0.002174  | 3551.437396 |   2.245583   | -16891.155853 |   -0.954286   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -61053.498659\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00014.pth\n",
      "Train Episode: [15/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "|  -688.419270% |   0.000683  | 12404.350216 |   2.003406   | 491928.343157 |    1.901195   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [15/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -170.679616% |  -0.002148  | 3483.353434 |   2.133488   | -17229.228497 |   -0.957504   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -60056.333731\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00015.pth\n",
      "Train Episode: [16/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "|  -972.226134% |   0.000515  | 15091.307919 |   2.283666   | 395975.151491 |    1.384039   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [16/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -230.134541% |  -0.001800  | 4108.651796 |   2.562529   | -14181.348608 |   -0.819081   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -56390.914075\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00016.pth\n",
      "Train Episode: [17/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| -1400.570121% |  -0.000281  | 36908.469387 |  29.389929   | -41092.361952 |   -0.717731   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [17/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -230.134541% |  -0.001800  | 4108.651796 |   2.562529   | -14181.348608 |   -0.819081   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -56390.914075\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00017.pth\n",
      "Train Episode: [18/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "|  -744.326652% |   0.000579  | 12140.578923 |   2.151154   | 379703.556119 |    1.557721   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [18/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -180.970136% |  -0.001587  | 3330.163550 |   2.008192   | -12932.140987 |   -0.734850   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -41071.900165\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00018.pth\n",
      "Train Episode: [19/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "|  -948.403058% |  -0.000274  | 24082.344472 |  19.543640   | -39251.932592 |   -0.693784   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [19/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -230.134541% |  -0.001800  | 4108.651796 |   2.562529   | -14181.348608 |   -0.819081   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -56390.914075\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00019.pth\n",
      "Train Episode: [20/20]\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility  | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "| -1874.527862% |  -0.000289  | 51213.539849 |  40.429782   | -42627.916754 |   -0.738897   |\n",
      "+---------------+-------------+--------------+--------------+---------------+---------------+\n",
      "Valid Episode: [20/20]\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|  -133.446994% |  -0.002186  | 3066.785470 |   1.822390   | -18069.448586 |   -0.946982   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Valid Episode Reward Sum: -54479.743152\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/checkpoint-00020.pth\n",
      "save path /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/best.pth\n",
      "Resume checkpoint /data1/sunshuo/qml/TradeMaster/TradeMasterReBuild/work_dir/algorithmic_trading_BTC_dqn_dqn_adam_mse/checkpoints/best.pth\n",
      "Test Best Episode\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "| Profit Margin | Sharp Ratio |  Volatility | Max Drawdown |  Calmar Ratio | Sortino Ratio |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "|   49.996525%  |  -0.005364  | 4336.971275 |   1.165761   | -98033.866998 |   -1.900394   |\n",
      "+---------------+-------------+-------------+--------------+---------------+---------------+\n",
      "Test Best Episode Reward Sum: -231100.467385\n",
      "train end\n"
     ]
    }
   ],
   "source": [
    "if task_name.startswith(\"train\"):\n",
    "    trainer.train_and_valid()\n",
    "    trainer.test()\n",
    "    print(\"train end\")\n",
    "elif task_name.startswith(\"test\"):\n",
    "    trainer.test()\n",
    "    print(\"test end\")\n",
    "elif task_name.startswith(\"style_test\"):\n",
    "    daily_return_list = []\n",
    "    for trainer in trainers:\n",
    "        daily_return_list.extend(trainer.test())\n",
    "    print('win rate is: ', sum(r > 0 for r in daily_return_list) / len(daily_return_list))\n",
    "    print(\"style test end\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "HFT",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.15"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "c33605b009166d65f90ad63d824c8e63d22d0973c031452c4b4158e2872c99ad"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
